Home Explore Help
Register Sign In
J1X
/
huginn
Watch 2
Star 0
Fork 0

Browse Source
Add multipart file upload to PostAgent

This allows the PostAgent to receive `file_pointer` events and upload the data to a website.

Dominik Sander 7 years ago
parent
4fdd69c6f4
commit
663eebc080
5 changed files with 98 additions and 34 deletions
Show Diff Stats
  1. 11 3
      app/concerns/file_handling.rb
  2. 1 0
      app/concerns/web_request_concern.rb
  3. 45 28
      app/models/agents/post_agent.rb
  4. 18 0
      spec/models/agents/post_agent_spec.rb
  5. 23 3
      spec/support/shared_examples/file_handling_consumer.rb

+ 11 - 3
app/concerns/file_handling.rb
View File 

@@ -5,13 +5,21 @@ module FileHandling
5 5 
     { file_pointer: { file: file, agent_id: id } }
6 6 
   end
7 7
8 
+  def has_file_pointer?(event)
9 
+    event.payload['file_pointer'] &&
10 
+      event.payload['file_pointer']['file'] &&
11 
+      event.payload['file_pointer']['agent_id']
12 
+  end
13 
+
8 14 
   def get_io(event)
9 
-    return nil unless event.payload['file_pointer'] &&
10 
-                      event.payload['file_pointer']['file'] &&
11 
-                      event.payload['file_pointer']['agent_id']
15 
+    return nil unless has_file_pointer?(event)
12 16 
     event.user.agents.find(event.payload['file_pointer']['agent_id']).get_io(event.payload['file_pointer']['file'])
13 17 
   end
14 18
19 
+  def get_upload_io(event)
20 
+    Faraday::UploadIO.new(get_io(event), MIME::Types.type_for(File.basename(event.payload['file_pointer']['file'])).first.try(:content_type))
21 
+  end
22 
+
15 23 
   def emitting_file_handling_agent_description
16 24 
     @emitting_file_handling_agent_description ||=
17 25 
       "This agent only emits a 'file pointer', not the data inside the files, the following agents can consume the created events: `#{receiving_file_handling_agents.join('`, `')}`. Read more about the concept in the [wiki](https://github.com/cantino/huginn/wiki/How-Huginn-works-with-files)."

+ 1 - 0
app/concerns/web_request_concern.rb
View File 

@@ -113,6 +113,7 @@ module WebRequestConcern
113 113 
       unless boolify(interpolated['disable_redirect_follow'])
114 114 
         builder.use FaradayMiddleware::FollowRedirects
115 115 
       end
116 
+      builder.request :multipart
116 117 
       builder.request :url_encoded
117 118
118 119 
       if boolify(interpolated['disable_url_encoding'])

+ 45 - 28
app/models/agents/post_agent.rb
View File 

@@ -1,6 +1,9 @@
1 1 
 module Agents
2 2 
   class PostAgent < Agent
3 3 
     include WebRequestConcern
4 
+    include FileHandling
5 
+
6 
+    consumes_file_pointer!
4 7
5 8 
     MIME_RE = /\A\w+\/.+\z/
6 9 
 
@@ -8,38 +11,44 @@ module Agents
8 11 
     no_bulk_receive!
9 12 
     default_schedule "never"
10 13
11 
-    description <<-MD
12 
-      A Post Agent receives events from other agents (or runs periodically), merges those events with the [Liquid-interpolated](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) contents of `payload`, and sends the results as POST (or GET) requests to a specified url.  To skip merging in the incoming event, but still send the interpolated payload, set `no_merge` to `true`.
14 
+    description do
15 
+      <<-MD
16 
+        A Post Agent receives events from other agents (or runs periodically), merges those events with the [Liquid-interpolated](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) contents of `payload`, and sends the results as POST (or GET) requests to a specified url.  To skip merging in the incoming event, but still send the interpolated payload, set `no_merge` to `true`.
13 17
14 
-      The `post_url` field must specify where you would like to send requests. Please include the URI scheme (`http` or `https`).
18 
+        The `post_url` field must specify where you would like to send requests. Please include the URI scheme (`http` or `https`).
15 19
16 
-      The `method` used can be any of `get`, `post`, `put`, `patch`, and `delete`.
20 
+        The `method` used can be any of `get`, `post`, `put`, `patch`, and `delete`.
17 21
18 
-      By default, non-GETs will be sent with form encoding (`application/x-www-form-urlencoded`).
22 
+        By default, non-GETs will be sent with form encoding (`application/x-www-form-urlencoded`).
19 23
20 
-      Change `content_type` to `json` to send JSON instead.
24 
+        Change `content_type` to `json` to send JSON instead.
21 25
22 
-      Change `content_type` to `xml` to send XML, where the name of the root element may be specified using `xml_root`, defaulting to `post`.
26 
+        Change `content_type` to `xml` to send XML, where the name of the root element may be specified using `xml_root`, defaulting to `post`.
23 27
24 
-      When `content_type` contains a [MIME](https://en.wikipedia.org/wiki/Media_type) type, and `payload` is a string, its interpolated value will be sent as a string in the HTTP request's body and the request's `Content-Type` HTTP header will be set to `content_type`. When `payload` is a string `no_merge` has to be set to `true`.
28 
+        When `content_type` contains a [MIME](https://en.wikipedia.org/wiki/Media_type) type, and `payload` is a string, its interpolated value will be sent as a string in the HTTP request's body and the request's `Content-Type` HTTP header will be set to `content_type`. When `payload` is a string `no_merge` has to be set to `true`.
25 29
26 
-      If `emit_events` is set to `true`, the server response will be emitted as an Event and can be fed to a WebsiteAgent for parsing (using its `data_from_event` and `type` options). No data processing
27 
-      will be attempted by this Agent, so the Event's "body" value will always be raw text.
28 
-      The Event will also have a "headers" hash and a "status" integer value.
29 
-      Set `event_headers_style` to one of the following values to normalize the keys of "headers" for downstream agents' convenience:
30 
+        If `emit_events` is set to `true`, the server response will be emitted as an Event and can be fed to a WebsiteAgent for parsing (using its `data_from_event` and `type` options). No data processing
31 
+        will be attempted by this Agent, so the Event's "body" value will always be raw text.
32 
+        The Event will also have a "headers" hash and a "status" integer value.
33 
+        Set `event_headers_style` to one of the following values to normalize the keys of "headers" for downstream agents' convenience:
30 34
31 
-        * `capitalized` (default) - Header names are capitalized; e.g. "Content-Type"
32 
-        * `downcased` - Header names are downcased; e.g. "content-type"
33 
-        * `snakecased` - Header names are snakecased; e.g. "content_type"
34 
-        * `raw` - Backward compatibility option to leave them unmodified from what the underlying HTTP library returns.
35 
+          * `capitalized` (default) - Header names are capitalized; e.g. "Content-Type"
36 
+          * `downcased` - Header names are downcased; e.g. "content-type"
37 
+          * `snakecased` - Header names are snakecased; e.g. "content_type"
38 
+          * `raw` - Backward compatibility option to leave them unmodified from what the underlying HTTP library returns.
35 39
36 
-      Other Options:
40 
+        Other Options:
37 41
38 
-        * `headers` - When present, it should be a hash of headers to send with the request.
39 
-        * `basic_auth` - Specify HTTP basic auth parameters: `"username:password"`, or `["username", "password"]`.
40 
-        * `disable_ssl_verification` - Set to `true` to disable ssl verification.
41 
-        * `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}").
42 
-    MD
42 
+          * `headers` - When present, it should be a hash of headers to send with the request.
43 
+          * `basic_auth` - Specify HTTP basic auth parameters: `"username:password"`, or `["username", "password"]`.
44 
+          * `disable_ssl_verification` - Set to `true` to disable ssl verification.
45 
+          * `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}").
46 
+
47 
+        #{receiving_file_handling_agent_description}
48 
+
49 
+        When receiving a `file_pointer` the request will be sent with multipart encoding (`multipart/form-data`) and `content_type` is ignored. `upload_key` can be used to specify the parameter in which the file will be sent, it defaults to `file`.
50 
+      MD
51 
+    end
43 52
44 53 
     event_description <<-MD
45 54 
       Events look like this:
 
@@ -125,9 +134,9 @@ module Agents
125 134 
         interpolate_with(event) do
126 135 
           outgoing = interpolated['payload'].presence || {}
127 136 
           if boolify(interpolated['no_merge'])
128 
-            handle outgoing, event.payload, headers(interpolated[:headers])
137 
+            handle outgoing, event, headers(interpolated[:headers])
129 138 
           else
130 
-            handle outgoing.merge(event.payload), event.payload, headers(interpolated[:headers])
139 
+            handle outgoing.merge(event.payload), event, headers(interpolated[:headers])
131 140 
           end
132 141 
         end
133 142 
       end
 
@@ -162,8 +171,8 @@ module Agents
162 171 
       }
163 172 
     end
164 173
165 
-    def handle(data, payload = {}, headers)
166 
-      url = interpolated(payload)[:post_url]
174 
+    def handle(data, event = Event.new, headers)
175 
+      url = interpolated(event.payload)[:post_url]
167 176
168 177 
       case method
169 178 
       when 'get', 'delete'
 
@@ -171,13 +180,21 @@ module Agents
171 180 
       when 'post', 'put', 'patch'
172 181 
         params = nil
173 182
174 
-        case (content_type = interpolated(payload)['content_type'])
183 
+        content_type =
184 
+          if has_file_pointer?(event)
185 
+            data[interpolated(event.payload)['upload_key'].presence || 'file'] = get_upload_io(event)
186 
+            nil
187 
+          else
188 
+            interpolated(event.payload)['content_type']
189 
+          end
190 
+
191 
+        case content_type
175 192 
         when 'json'
176 193 
           headers['Content-Type'] = 'application/json; charset=utf-8'
177 194 
           body = data.to_json
178 195 
         when 'xml'
179 196 
           headers['Content-Type'] = 'text/xml; charset=utf-8'
180 
-          body = data.to_xml(root: (interpolated(payload)[:xml_root] || 'post'))
197 
+          body = data.to_xml(root: (interpolated(event.payload)[:xml_root] || 'post'))
181 198 
         when MIME_RE
182 199 
           headers['Content-Type'] = content_type
183 200 
           body = data.to_s

+ 18 - 0
spec/models/agents/post_agent_spec.rb
View File 

@@ -57,6 +57,11 @@ describe Agents::PostAgent do
57 57 
   end
58 58
59 59 
   it_behaves_like WebRequestConcern
60 
+  it_behaves_like 'FileHandlingConsumer'
61 
+
62 
+  it 'renders the description markdown without errors' do
63 
+    expect { @checker.description }.not_to raise_error
64 
+  end
60 65
61 66 
   describe "making requests" do
62 67 
     it "can make requests of each type" do
 
@@ -149,6 +154,19 @@ describe Agents::PostAgent do
149 154 
       headers = @sent_requests[:post].first.headers
150 155 
       expect(headers["Foo"]).to eq("a_variable")
151 156 
     end
157 
+
158 
+    it 'makes a multipart request when receiving a file_pointer' do
159 
+      WebMock.reset!
160 
+      stub_request(:post, "http://www.example.com/").
161 
+        with(:body => "-------------RubyMultipartPost\r\nContent-Disposition: form-data; name=\"default\"\r\n\r\nvalue\r\n-------------RubyMultipartPost\r\nContent-Disposition: form-data; name=\"file\"; filename=\"local.path\"\r\nContent-Length: 8\r\nContent-Type: \r\nContent-Transfer-Encoding: binary\r\n\r\ntestdata\r\n-------------RubyMultipartPost--\r\n\r\n",
162 
+             :headers => {'Accept-Encoding'=>'gzip,deflate', 'Content-Length'=>'307', 'Content-Type'=>'multipart/form-data; boundary=-----------RubyMultipartPost', 'User-Agent'=>'Huginn - https://github.com/cantino/huginn'}).
163 
+        to_return(:status => 200, :body => "", :headers => {})
164 
+      event = Event.new(payload: {file_pointer: {agent_id: 111, file: 'test'}})
165 
+      io_mock = mock()
166 
+      mock(@checker).get_io(event) { StringIO.new("testdata") }
167 
+      @checker.options['no_merge'] = true
168 
+      @checker.receive([event])
169 
+    end
152 170 
   end
153 171
154 172 
   describe "#check" do

+ 23 - 3
spec/support/shared_examples/file_handling_consumer.rb
View File 

@@ -1,6 +1,8 @@
1 1 
 require 'rails_helper'
2 2
3 3 
 shared_examples_for 'FileHandlingConsumer' do
4 
+  let(:event) { Event.new(user: @checker.user, payload: {'file_pointer' => {'file' => 'text.txt', 'agent_id' => @checker.id}}) }
5 
+
4 6 
   it 'returns a file pointer' do
5 7 
     expect(@checker.get_file_pointer('testfile')).to eq(file_pointer: { file: "testfile", agent_id: @checker.id})
6 8 
   end
 
@@ -9,8 +11,26 @@ shared_examples_for 'FileHandlingConsumer' do
9 11 
     @checker2 = @checker.dup
10 12 
     @checker2.user = users(:bob)
11 13 
     @checker2.save!
12 
-    expect(@checker2.user.id).not_to eq(@checker.user.id)
13 
-    event = Event.new(user: @checker.user, payload: {'file_pointer' => {'file' => 'test', 'agent_id' => @checker2.id}})
14 
+    event.payload['file_pointer']['agent_id'] = @checker2.id
14 15 
     expect { @checker.get_io(event) }.to raise_error(ActiveRecord::RecordNotFound)
15 16 
   end
16 
-end
17 
+
18 
+  context '#has_file_pointer?' do
19 
+    it 'returns true if the event contains a file pointer' do
20 
+      expect(@checker.has_file_pointer?(event)).to be_truthy
21 
+    end
22 
+
23 
+    it 'returns false if the event does not contain a file pointer' do
24 
+      expect(@checker.has_file_pointer?(Event.new)).to be_falsy
25 
+    end
26 
+  end
27 
+
28 
+  it '#get_upload_io returns a Faraday::UploadIO instance' do
29 
+    io_mock = mock()
30 
+    mock(@checker).get_io(event) { StringIO.new("testdata") }
31 
+
32 
+    upload_io = @checker.get_upload_io(event)
33 
+    expect(upload_io).to be_a(Faraday::UploadIO)
34 
+    expect(upload_io.content_type).to eq('text/plain')
35 
+  end
36 
+end